﻿using myDocumentClustering.PreProcessing;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace myDocumentClustering.DocumentTermMatrix
{
    /// <summary>
    /// Suppose the documents to be clustered have been stored as text files.
    /// The text files don't need to be pre-processed.
    /// DTM with no n-gram
    /// </summary>
    public class DTMCreator
    {
        Vocabulary vocabulary = new Vocabulary(); // <word, word_index> this dict will grow forever.
        DTV dtVector;//document-term vector <word/col_index, word_count>
        public Documents docs = new Documents(); //documents to be clustered. <doc_name, doc_index>, doc_index start from 0.
        public DTM dtMatrix = new DTM(); //doc-term matrix
       

        const byte SPACE = 0x20;
        const Int32 assumedMaxWordLength = 256; // we assume the maximum length of a single word is <= 256.
        Byte[] wordBuffer = new Byte[assumedMaxWordLength]; // a word is defined as the continuous characters between 2 spaces.

        byte[] charMap = {//map all char to lower letters or space
	        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, //spaces  8
	        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, //spaces  16
	        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, //spaces  24
	        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, //spaces  32
	        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, //spaces  40
	        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, //spaces  48
	        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, //numbers  56
	        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, //numbers and spaces  64
	        0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, //upper case to lower case 72
	        0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, //upper case to lower case 80
	        0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, //upper case to lower case 88
	        0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, //96
	        0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, //this must be lower case 104
	        0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, //lower case 112
	        0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, //120
	        0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20  //128
        };


        public DTMCreator()
        {

        }

        public void LoadFilesRecursively(String docFolder)
        {
            String[] files = Directory.GetFiles(docFolder);
            foreach(var file in files)
            {
                docs[file] = docs.Count; // doc file Id start from 0;
            }
            String[] dirs = Directory.GetDirectories(docFolder);
            if (dirs.Length > 0)
            {
                foreach (var dir in dirs)
                    LoadFilesRecursively(dir);
            }
        }

        /// <summary>
        /// What can be seen as a valid term?
        /// </summary>
        /// <param name="useStemming"></param>
        public void GenerateDTM(Boolean useStemming, Boolean normalize, Boolean probalisitic, Boolean checkVector)
        {
            Encoding enc = Encoding.ASCII;

            foreach (var doc in docs.DocNames)
            {
                dtVector = new DTV();
                int wordLength = 0;

                FileStream fs = File.Open(doc, FileMode.Open);
                Byte[] bytes = new Byte[fs.Length];
                fs.Read(bytes, 0, (int)fs.Length);


                foreach (var rawByte in bytes)
                {
                    if (rawByte < 128)
                    {
                        byte mappedByte = charMap[rawByte];
                        if (mappedByte != SPACE)
                        {
                            wordBuffer[wordLength] = mappedByte;
                            wordLength++;
                        }
                        else
                        {
                            String word = enc.GetString(wordBuffer, 0, wordLength);
                            if (wordLength > 0 && !IsStopWord(word))
                            {
                                if (useStemming)
                                {
                                    //TBD
                                }
                                else
                                {
                                    Int32 wordColNumber = -1;
                                    if (vocabulary.Contains(word))
                                    {
                                        wordColNumber = vocabulary[word];
                                    }
                                    else
                                    {
                                        wordColNumber = vocabulary.Count + 1;
                                        vocabulary[word] = wordColNumber;
                                    }
                                    dtVector.AddWord(wordColNumber);
                                }
                            }
                            wordLength = 0;
                        }//word processed
                    }
                    fs.Close();
                }//file processed, raw vector obtained.

                if(normalize)
                    dtVector.NormalizeIt();
                if (probalisitic)
                    dtVector.MakeItProbalistic();
                Int32 docIndex = docs[doc];
                dtVector.docIndex = docIndex;                
                dtMatrix[docIndex] = dtVector;

                if (checkVector)
                {
                    OutputVector(dtVector);
                }
            }
            vocabulary.CreateInverseIndex();
        }

        public void OutputVector(DTV v)
        {
            Int32 col = 0;
            foreach (var key in v.vector.Keys)
            {
                col++;
                Console.Write("{0}-{1}, ", vocabulary[key], v.vector[key]);
                if (col % 8 == 0)
                    Console.WriteLine();
            }
            Console.WriteLine();
            Console.WriteLine("{0}", docs[v.docIndex]);
            Console.WriteLine();
            Console.ReadLine();
        }

        public Boolean IsStopWord(String word)
        {
            return StopWordFilter.IsStopWord(word);
        }

        public void OutputDTM()
        { }

    }

    /// <summary>
    /// DTM with n-gram
    /// </summary>
    public class DTM_NGram
    {
        public DTM_NGram()
        {}
    }
}
